From fd84e97f49c443f0c353185dc12ecdb06e36259f Mon Sep 17 00:00:00 2001 From: Taylor Hawkes Date: Thu, 15 Oct 2026 09:17:36 -0600 Subject: [PATCH] feat(providers): add Inference APIs as OpenAI-compatible provider (#41669) * feat(providers): add Inference APIs as OpenAI-compatible provider Registers https://api.inferenceapis.com/v1 as a JSON-configured provider (chat completions and Responses API). Co-Authored-By: Claude Fable 5.1 * fix(providers): list Inference APIs in Add Model and the provider docs index, add request-level tests - provider_create_fields.json entry so the Add Model dropdown test passes - provider_endpoints_support.json (+ backup) entry so the provider docs check passes - tests now capture the outgoing HTTP request (URL, auth header, body) instead of only inspecting registrations - drop a redundant comment Co-Authored-By: Claude Fable 5.1 * test(providers): inject a recording HTTP session instead of patching httpx.Client Request-level tests now set litellm.client_session to an httpx.Client with a MockTransport, so nothing is monkeypatched on a class. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011qUBY2P8qTTuJCUeMzTCD4 * chore(providers): drop redundant comment on the inferenceapis entry Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011qUBY2P8qTTuJCUeMzTCD4 * feat(inferenceapis): add cost map rows for the 12 chat models and test the responses route Prices and context windows come from https://inferenceapis.com/pricing and GET /v1/models. Moves the provider test to tests/unit and adds the interactions flag the endpoint matrix now carries. * feat(inferenceapis): price the vendor's chat model aliases and mark /v1/messages supported * fix(inferenceapis): serve /v1/messages natively and keep audio routes off the provider * fix(json providers): send provider-specific params in extra_body and report context overflow as ContextWindowExceededError * fix(exception mapping): match the Inference APIs overflow phrase instead of any context_length_exceeded substring * fix(exception mapping): map context overflow to ContextWindowExceededError for Inference APIs only Providers without their own mapper keep reporting an overflow as BadRequestError, since the router raises a ContextWindowExceededError instead of trying generic fallbacks when context_window_fallbacks has no entry for the group. Adds the integration cells for the provider, its fallbacks, and the extra_body and overflow families. * test(inferenceapis): hold upstream replies across the worker kill in the chaos cell A reply the killed worker sent before SIGKILL can lose its queued spend row, which the product never promised to keep. Holding the upstream until after the kill means only the surviving worker answers, so every 200 must log exactly once. * test(inferenceapis): make the chaos cell kill the quieter worker and always release held replies The survivor now must hold at least one in-flight request, so the burst check cannot pass as 0 == 0, and the held vendor is released on exit so a failed assertion cannot stall the wire server teardown --------- Co-authored-by: TaylorHawkes <857858+TaylorHawkes@users.noreply.github.com> Co-authored-by: Claude Fable 5.1 Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/constants.py | 1 + .../exception_mapping_utils.py | 3 + litellm/llms/openai_like/providers.json | 10 + ...odel_prices_and_context_window_backup.json | 730 ++++++++++++++++++ .../provider_endpoints_support_backup.json | 18 + .../provider_create_fields.json | 28 + litellm/types/utils.py | 1 + litellm/utils.py | 9 +- model_prices_and_context_window.json | 730 ++++++++++++++++++ provider_endpoints_support.json | 18 + ...renceapis_context_window_fallbacks_wire.py | 400 ++++++++++ .../providers/test_inferenceapis_wire.py | 580 ++++++++++++++ ...ider_error_and_extra_body_families_wire.py | 304 ++++++++ .../test_exception_mapping_utils.py | 14 + .../test_inferenceapis_provider.py | 272 +++++++ 15 files changed, 3117 insertions(+), 1 deletion(-) create mode 100644 tests/integration/providers/test_inferenceapis_context_window_fallbacks_wire.py create mode 100644 tests/integration/providers/test_inferenceapis_wire.py create mode 100644 tests/integration/sdk/test_provider_error_and_extra_body_families_wire.py create mode 100644 tests/unit/llms/openai_like/test_inferenceapis_provider.py diff --git a/litellm/constants.py b/litellm/constants.py index 274854e1c40..6e1b5baad13 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1131,6 +1131,7 @@ _openai_like_providers: Final[list[str]] = [ "databricks", "lemonade", "watsonx", + "inferenceapis", ] # private helper. similar to openai but require some custom auth / endpoint handling, so can't use the openai sdk # well supported replicate llms replicate_models: Final[set] = set( diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 014f0cae54f..82d3ea0adb9 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -113,6 +113,9 @@ class ExceptionCheckers: if "maximum input length is" in _error_str_lowercase and "tokens" in _error_str_lowercase: return True + if "input too long:" in _error_str_lowercase and "input tokens, limit is" in _error_str_lowercase: + return True + return False @staticmethod diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index d25257b0e81..1a987595da2 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -229,5 +229,15 @@ "base_url": "https://api.reka.ai/v1", "api_key_env": "REKA_API_KEY", "api_base_env": "REKA_API_BASE" + }, + "inferenceapis": { + "base_url": "https://api.inferenceapis.com/v1", + "api_key_env": "INFERENCEAPIS_API_KEY", + "api_base_env": "INFERENCEAPIS_API_BASE", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ] } } diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 1cdad96a076..cdec1c2f449 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -39144,6 +39144,736 @@ "supports_system_messages": true, "supports_tool_choice": true }, + "inferenceapis/deepseek-ai/DeepSeek-V3": { + "cache_read_input_token_cost": 4e-08, + "input_cost_per_token": 1.9e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 3.8e-07, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/deepseek-ai/DeepSeek-V4-Flash": { + "cache_read_input_token_cost": 4e-08, + "input_cost_per_token": 1.9e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 3.8e-07, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/deepseek-ai/DeepSeek-V4-Flash-0731": { + "cache_read_input_token_cost": 4e-08, + "input_cost_per_token": 1.9e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 3.8e-07, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/deepseek-ai/DeepSeek-V4-Pro": { + "cache_read_input_token_cost": 1.7e-07, + "input_cost_per_token": 1.72e-06, + "litellm_provider": "inferenceapis", + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 5.15e-06, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/deepseek-ai/DeepSeek-V4-Pro-0813": { + "cache_read_input_token_cost": 1.7e-07, + "input_cost_per_token": 1.72e-06, + "litellm_provider": "inferenceapis", + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 5.15e-06, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/deepseek-ai/DeepSeek-V4.1-Flash": { + "cache_read_input_token_cost": 1e-08, + "input_cost_per_token": 4e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 1.6e-06, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/deepseek-chat": { + "cache_read_input_token_cost": 4e-08, + "input_cost_per_token": 1.9e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 3.8e-07, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/deepseek-flash": { + "cache_read_input_token_cost": 1e-08, + "input_cost_per_token": 4e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 1.6e-06, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/deepseek-reasoner": { + "cache_read_input_token_cost": 4e-08, + "input_cost_per_token": 1.9e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 3.8e-07, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/deepseek-v3": { + "cache_read_input_token_cost": 4e-08, + "input_cost_per_token": 1.9e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 3.8e-07, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/deepseek-v4-flash": { + "cache_read_input_token_cost": 4e-08, + "input_cost_per_token": 1.9e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 3.8e-07, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/deepseek-v4-pro": { + "cache_read_input_token_cost": 1.7e-07, + "input_cost_per_token": 1.72e-06, + "litellm_provider": "inferenceapis", + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 5.15e-06, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/deepseek-v4.1-flash": { + "cache_read_input_token_cost": 1e-08, + "input_cost_per_token": 4e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 1.6e-06, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/glm-5.3": { + "cache_read_input_token_cost": 3.4e-07, + "input_cost_per_token": 1.82e-06, + "litellm_provider": "inferenceapis", + "max_input_tokens": 1048575, + "max_tokens": 1048575, + "mode": "chat", + "output_cost_per_token": 5.72e-06, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/glm-5.3-flash": { + "cache_read_input_token_cost": 4e-08, + "input_cost_per_token": 2e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 1048575, + "max_tokens": 1048575, + "mode": "chat", + "output_cost_per_token": 6.6e-07, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/gpt-oss-120b": { + "input_cost_per_token": 2e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 8e-07, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/gpt-oss-20b": { + "input_cost_per_token": 4e-08, + "litellm_provider": "inferenceapis", + "max_input_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1.9e-07, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/kimi-k2.7-code": { + "cache_read_input_token_cost": 1.8e-07, + "input_cost_per_token": 8.9e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 4.42e-06, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/llama-3.1-8b-instant": { + "input_cost_per_token": 2e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 5e-07, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_reasoning": false, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/llama-3.3-70b": { + "input_cost_per_token": 2e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 5e-07, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_reasoning": false, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/llama-3.3-70b-instruct": { + "input_cost_per_token": 2e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 5e-07, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_reasoning": false, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/llama-3.3-70b-versatile": { + "input_cost_per_token": 2e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 5e-07, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_reasoning": false, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/meta-llama/Llama-3.3-70B-Instruct": { + "input_cost_per_token": 2e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 5e-07, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_reasoning": false, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/meta-llama/Llama-3.3-70B-Instruct-Turbo": { + "input_cost_per_token": 2e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 5e-07, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_reasoning": false, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/minimax-m3": { + "cache_read_input_token_cost": 8e-08, + "input_cost_per_token": 3.9e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 524288, + "max_tokens": 524288, + "mode": "chat", + "output_cost_per_token": 1.56e-06, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/MiniMaxAI/MiniMax-M3": { + "cache_read_input_token_cost": 8e-08, + "input_cost_per_token": 3.9e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 524288, + "max_tokens": 524288, + "mode": "chat", + "output_cost_per_token": 1.56e-06, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/moonshotai/Kimi-K2.7-Code": { + "cache_read_input_token_cost": 1.8e-07, + "input_cost_per_token": 8.9e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 4.42e-06, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/openai/gpt-oss-120b": { + "input_cost_per_token": 2e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 8e-07, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/openai/gpt-oss-20b": { + "input_cost_per_token": 4e-08, + "litellm_provider": "inferenceapis", + "max_input_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1.9e-07, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/Qwen/Qwen3-VL-235B-A22B-Instruct": { + "input_cost_per_token": 2.6e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 1.15e-06, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_reasoning": false, + "supports_tool_choice": true, + "supports_vision": true + }, + "inferenceapis/Qwen/Qwen3.8-Flash": { + "cache_read_input_token_cost": 2e-08, + "input_cost_per_token": 1.5e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 5e-07, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/qwen3-vl": { + "input_cost_per_token": 2.6e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 1.15e-06, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_reasoning": false, + "supports_tool_choice": true, + "supports_vision": true + }, + "inferenceapis/qwen3-vl-235b": { + "input_cost_per_token": 2.6e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 1.15e-06, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_reasoning": false, + "supports_tool_choice": true, + "supports_vision": true + }, + "inferenceapis/qwen3.8-flash": { + "cache_read_input_token_cost": 2e-08, + "input_cost_per_token": 1.5e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 5e-07, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/zai-org/GLM-5.3": { + "cache_read_input_token_cost": 3.4e-07, + "input_cost_per_token": 1.82e-06, + "litellm_provider": "inferenceapis", + "max_input_tokens": 1048575, + "max_tokens": 1048575, + "mode": "chat", + "output_cost_per_token": 5.72e-06, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/zai-org/GLM-5.3-Flash": { + "cache_read_input_token_cost": 4e-08, + "input_cost_per_token": 2e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 1048575, + "max_tokens": 1048575, + "mode": "chat", + "output_cost_per_token": 6.6e-07, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, "text-completion-inception/mercury-edit-2": { "cache_read_input_token_cost": 2.5e-08, "input_cost_per_token": 2.5e-07, diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json index 8dbdac37717..a10f2f93d48 100644 --- a/litellm/provider_endpoints_support_backup.json +++ b/litellm/provider_endpoints_support_backup.json @@ -2189,6 +2189,24 @@ "a2a": false } }, + "inferenceapis": { + "display_name": "Inference APIs (`inferenceapis`)", + "url": "https://docs.litellm.ai/docs/providers/inferenceapis", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false, + "interactions": false + } + }, "snowflake": { "display_name": "Snowflake (`snowflake`)", "url": "https://docs.litellm.ai/docs/providers/snowflake", diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index 86510a5678c..4a6a88f9a86 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -3323,6 +3323,34 @@ ], "default_model_placeholder": "scx-ai/GLM-5.2" }, + { + "provider": "INFERENCEAPIS", + "provider_display_name": "Inference APIs", + "litellm_provider": "inferenceapis", + "credential_fields": [ + { + "key": "api_base", + "label": "API Base", + "placeholder": "https://api.inferenceapis.com/v1", + "tooltip": null, + "required": false, + "field_type": "text", + "options": null, + "default_value": null + }, + { + "key": "api_key", + "label": "API Key", + "placeholder": null, + "tooltip": null, + "required": true, + "field_type": "password", + "options": null, + "default_value": null + } + ], + "default_model_placeholder": "inferenceapis/deepseek-ai/DeepSeek-V4-Flash" + }, { "provider": "Snowflake", "provider_display_name": "Snowflake", diff --git a/litellm/types/utils.py b/litellm/types/utils.py index c3eba945277..dafd9f5b771 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4337,6 +4337,7 @@ class LlmProviders(str, Enum): CORTECS = "cortecs" SCX_AI = "scx-ai" PRISM = "prism" + INFERENCEAPIS = "inferenceapis" DARKBLOOM = "darkbloom" META = "meta" SAIL = "sail" diff --git a/litellm/utils.py b/litellm/utils.py index 22aa4baf52e..22d52fe8d74 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5162,8 +5162,15 @@ def add_provider_specific_params_to_optional_params( """ Add provider specific params to optional_params """ + from litellm.llms.openai_like.json_loader import JSONProviderRegistry - if custom_llm_provider in ["openai", "azure", "text-completion-openai"] + litellm.openai_compatible_providers: + calls_through_openai_sdk: Final = custom_llm_provider in [ + "openai", + "azure", + "text-completion-openai", + *litellm.openai_compatible_providers, + ] or JSONProviderRegistry.exists(custom_llm_provider) + if calls_through_openai_sdk: # for openai, azure we should pass the extra/passed params within `extra_body` https://github.com/openai/openai-python/blob/ac33853ba10d13ac149b1fa3ca6dba7d613065c9/src/openai/resources/models.py#L46 if _should_drop_param(k="extra_body", additional_drop_params=additional_drop_params) is False: extra_body: Final = dict(passed_params.pop("extra_body", None) or {}) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 1cdad96a076..cdec1c2f449 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -39144,6 +39144,736 @@ "supports_system_messages": true, "supports_tool_choice": true }, + "inferenceapis/deepseek-ai/DeepSeek-V3": { + "cache_read_input_token_cost": 4e-08, + "input_cost_per_token": 1.9e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 3.8e-07, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/deepseek-ai/DeepSeek-V4-Flash": { + "cache_read_input_token_cost": 4e-08, + "input_cost_per_token": 1.9e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 3.8e-07, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/deepseek-ai/DeepSeek-V4-Flash-0731": { + "cache_read_input_token_cost": 4e-08, + "input_cost_per_token": 1.9e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 3.8e-07, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/deepseek-ai/DeepSeek-V4-Pro": { + "cache_read_input_token_cost": 1.7e-07, + "input_cost_per_token": 1.72e-06, + "litellm_provider": "inferenceapis", + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 5.15e-06, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/deepseek-ai/DeepSeek-V4-Pro-0813": { + "cache_read_input_token_cost": 1.7e-07, + "input_cost_per_token": 1.72e-06, + "litellm_provider": "inferenceapis", + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 5.15e-06, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/deepseek-ai/DeepSeek-V4.1-Flash": { + "cache_read_input_token_cost": 1e-08, + "input_cost_per_token": 4e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 1.6e-06, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/deepseek-chat": { + "cache_read_input_token_cost": 4e-08, + "input_cost_per_token": 1.9e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 3.8e-07, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/deepseek-flash": { + "cache_read_input_token_cost": 1e-08, + "input_cost_per_token": 4e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 1.6e-06, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/deepseek-reasoner": { + "cache_read_input_token_cost": 4e-08, + "input_cost_per_token": 1.9e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 3.8e-07, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/deepseek-v3": { + "cache_read_input_token_cost": 4e-08, + "input_cost_per_token": 1.9e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 3.8e-07, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/deepseek-v4-flash": { + "cache_read_input_token_cost": 4e-08, + "input_cost_per_token": 1.9e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 3.8e-07, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/deepseek-v4-pro": { + "cache_read_input_token_cost": 1.7e-07, + "input_cost_per_token": 1.72e-06, + "litellm_provider": "inferenceapis", + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 5.15e-06, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/deepseek-v4.1-flash": { + "cache_read_input_token_cost": 1e-08, + "input_cost_per_token": 4e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 1.6e-06, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/glm-5.3": { + "cache_read_input_token_cost": 3.4e-07, + "input_cost_per_token": 1.82e-06, + "litellm_provider": "inferenceapis", + "max_input_tokens": 1048575, + "max_tokens": 1048575, + "mode": "chat", + "output_cost_per_token": 5.72e-06, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/glm-5.3-flash": { + "cache_read_input_token_cost": 4e-08, + "input_cost_per_token": 2e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 1048575, + "max_tokens": 1048575, + "mode": "chat", + "output_cost_per_token": 6.6e-07, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/gpt-oss-120b": { + "input_cost_per_token": 2e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 8e-07, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/gpt-oss-20b": { + "input_cost_per_token": 4e-08, + "litellm_provider": "inferenceapis", + "max_input_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1.9e-07, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/kimi-k2.7-code": { + "cache_read_input_token_cost": 1.8e-07, + "input_cost_per_token": 8.9e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 4.42e-06, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/llama-3.1-8b-instant": { + "input_cost_per_token": 2e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 5e-07, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_reasoning": false, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/llama-3.3-70b": { + "input_cost_per_token": 2e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 5e-07, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_reasoning": false, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/llama-3.3-70b-instruct": { + "input_cost_per_token": 2e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 5e-07, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_reasoning": false, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/llama-3.3-70b-versatile": { + "input_cost_per_token": 2e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 5e-07, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_reasoning": false, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/meta-llama/Llama-3.3-70B-Instruct": { + "input_cost_per_token": 2e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 5e-07, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_reasoning": false, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/meta-llama/Llama-3.3-70B-Instruct-Turbo": { + "input_cost_per_token": 2e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 5e-07, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_reasoning": false, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/minimax-m3": { + "cache_read_input_token_cost": 8e-08, + "input_cost_per_token": 3.9e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 524288, + "max_tokens": 524288, + "mode": "chat", + "output_cost_per_token": 1.56e-06, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/MiniMaxAI/MiniMax-M3": { + "cache_read_input_token_cost": 8e-08, + "input_cost_per_token": 3.9e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 524288, + "max_tokens": 524288, + "mode": "chat", + "output_cost_per_token": 1.56e-06, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/moonshotai/Kimi-K2.7-Code": { + "cache_read_input_token_cost": 1.8e-07, + "input_cost_per_token": 8.9e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 4.42e-06, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/openai/gpt-oss-120b": { + "input_cost_per_token": 2e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 8e-07, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/openai/gpt-oss-20b": { + "input_cost_per_token": 4e-08, + "litellm_provider": "inferenceapis", + "max_input_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1.9e-07, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/Qwen/Qwen3-VL-235B-A22B-Instruct": { + "input_cost_per_token": 2.6e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 1.15e-06, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_reasoning": false, + "supports_tool_choice": true, + "supports_vision": true + }, + "inferenceapis/Qwen/Qwen3.8-Flash": { + "cache_read_input_token_cost": 2e-08, + "input_cost_per_token": 1.5e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 5e-07, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/qwen3-vl": { + "input_cost_per_token": 2.6e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 1.15e-06, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_reasoning": false, + "supports_tool_choice": true, + "supports_vision": true + }, + "inferenceapis/qwen3-vl-235b": { + "input_cost_per_token": 2.6e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 1.15e-06, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_reasoning": false, + "supports_tool_choice": true, + "supports_vision": true + }, + "inferenceapis/qwen3.8-flash": { + "cache_read_input_token_cost": 2e-08, + "input_cost_per_token": 1.5e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 5e-07, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/zai-org/GLM-5.3": { + "cache_read_input_token_cost": 3.4e-07, + "input_cost_per_token": 1.82e-06, + "litellm_provider": "inferenceapis", + "max_input_tokens": 1048575, + "max_tokens": 1048575, + "mode": "chat", + "output_cost_per_token": 5.72e-06, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "inferenceapis/zai-org/GLM-5.3-Flash": { + "cache_read_input_token_cost": 4e-08, + "input_cost_per_token": 2e-07, + "litellm_provider": "inferenceapis", + "max_input_tokens": 1048575, + "max_tokens": 1048575, + "mode": "chat", + "output_cost_per_token": 6.6e-07, + "source": "https://inferenceapis.com/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, "text-completion-inception/mercury-edit-2": { "cache_read_input_token_cost": 2.5e-08, "input_cost_per_token": 2.5e-07, diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index b6c45fd4bb4..45104d40018 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -2460,6 +2460,24 @@ "a2a": false } }, + "inferenceapis": { + "display_name": "Inference APIs (`inferenceapis`)", + "url": "https://docs.litellm.ai/docs/providers/inferenceapis", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false, + "interactions": false + } + }, "snowflake": { "display_name": "Snowflake (`snowflake`)", "url": "https://docs.litellm.ai/docs/providers/snowflake", diff --git a/tests/integration/providers/test_inferenceapis_context_window_fallbacks_wire.py b/tests/integration/providers/test_inferenceapis_context_window_fallbacks_wire.py new file mode 100644 index 00000000000..04869ebb77f --- /dev/null +++ b/tests/integration/providers/test_inferenceapis_context_window_fallbacks_wire.py @@ -0,0 +1,400 @@ +import json +import threading +from collections.abc import Callable, Iterator, Mapping +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager +from dataclasses import dataclass +from functools import partial +from pathlib import Path +from queue import SimpleQueue +from types import MappingProxyType +from typing import Final +from uuid import uuid4 + +import httpx +import pytest +import yaml +from integration._support.bedrock_discovery import open_connections_to +from integration._support.client import Gateway, eventually, gateway_from_environment +from integration._support.database import read_rows +from integration._support.process import OwnedProxy, graceful_stop_seconds, owned_proxy_process +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +pytestmark = pytest.mark.timeout(int(2 * graceful_stop_seconds() + 120)) + +_BACKEND: Final = "deepseek-ai/DeepSeek-V4-Flash" +_VENDOR_KEY: Final = "synthetic-inferenceapis-key" +_OPENAI_KEY: Final = "synthetic-openai-key" +_INFERENCEAPIS_OVERFLOW: Final = "Input too long: 150055 input tokens, limit is 131072 for this model" +_STANDARD_OVERFLOW: Final = "This model's maximum context length is 4096 tokens. However, you requested 9000 tokens." +_PLAIN_TEXT: Final = "Invalid value for 'temperature': expected a number between 0 and 2." +_FALLBACK_TEXT: Final = "fallback answered" +_SLOW_PIECES: Final = 4 +_SLOW_PAUSE_SECONDS: Final = 0.5 +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_EMPTY_MODEL_LIST: Final = Reply(body=b'{"object":"list","data":[]}') + + +def _error_peer(message: str, code: str | None) -> Callable[[Request], Reply]: + body: Final = json.dumps({"error": {"message": message, "type": "invalid_request_error", "code": code}}).encode() + return lambda request: _EMPTY_MODEL_LIST if request.method == "GET" else Reply(status=400, body=body) + + +def _streaming(request: Request) -> bool: + return _JSON_OBJECT.validate_json(request.body).get("stream") is True + + +def _sse(events: tuple[Mapping[str, JsonValue], ...]) -> tuple[bytes, ...]: + return tuple(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode() for event in events) + + +def _completion(identity: str, text: str) -> bytes: + return json.dumps( + { + "id": identity, + "object": "chat.completion", + "created": 1, + "model": "gpt-4o-mini", + "choices": [{"index": 0, "message": {"role": "assistant", "content": text}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 9, "completion_tokens": 2, "total_tokens": 11}, + } + ).encode() + + +def _chat_reply(identity: str, stream: bool) -> Reply: + if not stream: + return Reply(body=_completion(identity, _FALLBACK_TEXT)) + chunk: Final[dict[str, JsonValue]] = { + "id": identity, + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-4o-mini", + } + frames: Final[tuple[dict[str, JsonValue], ...]] = ( + {**chunk, "choices": [{"index": 0, "delta": {"role": "assistant", "content": _FALLBACK_TEXT}}]}, + {**chunk, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}, + ) + return Reply( + content_type="text/event-stream", + chunks=tuple(f"data: {json.dumps(frame)}\n\n".encode() for frame in frames) + (b"data: [DONE]\n\n",), + ) + + +def _responses_reply(identity: str, stream: bool) -> Reply: + completed: Final[dict[str, JsonValue]] = { + "id": f"resp_{identity}", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gpt-4o-mini", + "output": [ + { + "type": "message", + "id": f"msg_{identity}", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": _FALLBACK_TEXT, "annotations": []}], + } + ], + "usage": {"input_tokens": 9, "output_tokens": 2, "total_tokens": 11}, + } + if not stream: + return Reply(body=json.dumps(completed).encode()) + return Reply( + content_type="text/event-stream", + chunks=_sse( + ( + { + "type": "response.created", + "sequence_number": 0, + "response": {**completed, "status": "in_progress", "output": []}, + }, + { + "type": "response.output_text.delta", + "sequence_number": 1, + "item_id": f"msg_{identity}", + "output_index": 0, + "content_index": 0, + "delta": _FALLBACK_TEXT, + }, + {"type": "response.completed", "sequence_number": 2, "response": completed}, + ) + ), + ) + + +def _fallback_peer(request: Request) -> Reply: + if request.method == "GET": + return _EMPTY_MODEL_LIST + assert request.headers["authorization"] == f"Bearer {_OPENAI_KEY}", dict(request.headers) + identity: Final = f"chatcmpl-fallback-{uuid4().hex}" + if request.target == "/v1/responses": + return _responses_reply(identity, _streaming(request)) + assert request.target == "/v1/chat/completions", request.target + return _chat_reply(identity, _streaming(request)) + + +def _healthy_peer(request: Request) -> Reply: + assert (request.method, request.target) == ("POST", "/v1/chat/completions"), request.target + return Reply(body=_completion(f"chatcmpl-healthy-{uuid4().hex}", "healthy")) + + +def _slow_peer(request: Request) -> Reply: + assert (request.method, request.target) == ("POST", "/v1/chat/completions"), request.target + body: Final = _completion(f"chatcmpl-slow-{uuid4().hex}", "slow") + size: Final = -(-len(body) // _SLOW_PIECES) + return Reply( + chunks=tuple(body[start : start + size] for start in range(0, len(body), size)), + pause_between_chunks=_SLOW_PAUSE_SECONDS, + ) + + +@dataclass(frozen=True, slots=True) +class _HeldPeer: + release: threading.Event + held: SimpleQueue[str] + + def respond(self, request: Request) -> Reply: + self.held.put(request.target) + assert self.release.wait(timeout=120), "The burst was never released" + return Reply(body=_completion(f"chatcmpl-held-{uuid4().hex}", "held")) + + @contextmanager + def released_on_exit(self) -> Iterator[None]: + try: + yield + finally: + self.release.set() + + +@dataclass(frozen=True, slots=True) +class Rig: + gateway: Gateway + owned: OwnedProxy + peers: Mapping[str, Wire] + models: Mapping[str, str] + held: _HeldPeer + + +def _vendor(provider_model: str, wire: Wire) -> dict[str, JsonValue]: + return {"model": provider_model, "api_base": f"{wire.url}/v1", "api_key": _VENDOR_KEY} + + +def _write_config(root: Path, model_list: list[JsonValue], router_settings: dict[str, JsonValue]) -> Path: + base: Final = _JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) + config: Final[dict[str, JsonValue]] = { + **base, + "model_list": model_list, + "router_settings": {"disable_cooldowns": True, "num_retries": 0, **router_settings}, + } + path: Final = root / "inferenceapis-fallbacks.yaml" + path.write_text(yaml.safe_dump(config)) + return path + + +@pytest.fixture(scope="module") +def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]: + root: Final = tmp_path_factory.mktemp("inferenceapis-fallbacks") + suffix: Final = uuid4().hex[:8] + names: Final = { + label: f"{label}-{suffix}" + for label in ("ia_cwf", "ia_plain", "mimo_fb", "ia_ok", "ia_slow", "ia_held", "fallback") + } + held: Final = _HeldPeer(threading.Event(), SimpleQueue()) + with ( + gateway_from_environment() as gateway, + wire_server(_error_peer(_INFERENCEAPIS_OVERFLOW, "upstream_rejected")) as ia_overflow, + wire_server(_error_peer(_PLAIN_TEXT, None)) as ia_plain, + wire_server(_error_peer(_STANDARD_OVERFLOW, None)) as mimo_overflow, + wire_server(_healthy_peer) as healthy, + wire_server(_slow_peer) as slow, + wire_server(held.respond) as held_wire, + wire_server(_fallback_peer) as fallback, + ): + model_list: Final[list[JsonValue]] = [ + {"model_name": names["ia_cwf"], "litellm_params": _vendor(f"inferenceapis/{_BACKEND}", ia_overflow)}, + {"model_name": names["ia_plain"], "litellm_params": _vendor(f"inferenceapis/{_BACKEND}", ia_plain)}, + {"model_name": names["mimo_fb"], "litellm_params": _vendor("xiaomi_mimo/mimo-v2.6-flash", mimo_overflow)}, + {"model_name": names["ia_ok"], "litellm_params": _vendor(f"inferenceapis/{_BACKEND}", healthy)}, + {"model_name": names["ia_slow"], "litellm_params": _vendor(f"inferenceapis/{_BACKEND}", slow)}, + {"model_name": names["ia_held"], "litellm_params": _vendor(f"inferenceapis/{_BACKEND}", held_wire)}, + { + "model_name": names["fallback"], + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_base": f"{fallback.url}/v1", + "api_key": _OPENAI_KEY, + }, + }, + ] + config: Final = _write_config( + root, + model_list, + { + "context_window_fallbacks": [{names[label]: [names["fallback"]]} for label in ("ia_cwf", "ia_plain")], + "fallbacks": [{names["mimo_fb"]: [names["fallback"]]}], + }, + ) + with owned_proxy_process(gateway, root, {}, config=config, workers=2) as owned: + yield Rig( + owned.gateway, + owned, + { + "ia_cwf": ia_overflow, + "ia_plain": ia_plain, + "mimo_fb": mimo_overflow, + "ia_ok": healthy, + "ia_slow": slow, + "ia_held": held_wire, + "fallback": fallback, + }, + names, + held, + ) + + +def _chat(model: str, prompt: str, stream: bool = False) -> dict[str, JsonValue]: + return {"model": model, "messages": [{"role": "user", "content": prompt}], "stream": stream} + + +def _messages(model: str, prompt: str, stream: bool = False) -> dict[str, JsonValue]: + return {"model": model, "max_tokens": 32, "messages": [{"role": "user", "content": prompt}], "stream": stream} + + +def _responses(model: str, prompt: str, stream: bool = False) -> dict[str, JsonValue]: + return {"model": model, "input": prompt, "stream": stream} + + +_BUILDERS: Final[Mapping[str, Callable[[str, str, bool], dict[str, JsonValue]]]] = { + "/v1/chat/completions": _chat, + "/v1/messages": _messages, + "/v1/responses": _responses, +} +_SURFACES: Final = tuple((route, stream) for route in _BUILDERS for stream in (False, True)) + + +def _consumed(gateway: Gateway, route: str, body: Mapping[str, JsonValue]) -> tuple[int, str]: + headers: Final = {"Authorization": f"Bearer {gateway.key}"} + with gateway.client.stream("POST", route, json=body, headers=headers) as response: + text: Final = b"".join(response.iter_bytes()).decode() + return response.status_code, text + + +def _prompt(label: str) -> str: + return f"{label} {uuid4().hex}" + + +def _calls_with(requests: tuple[Request, ...], prompt: str) -> int: + return sum(1 for request in requests if prompt in request.body.decode()) + + +def _send(rig: Rig, label: str, route: str, stream: bool, prompt: str) -> tuple[int, str]: + return _consumed(rig.gateway, route, _BUILDERS[route](rig.models[label], prompt, stream)) + + +@pytest.mark.parametrize(("route", "stream"), _SURFACES) +def test_inferenceapis_overflow_is_answered_by_the_context_window_fallback(rig: Rig, route: str, stream: bool) -> None: + prompt: Final = _prompt("ia cwf") + status, text = _send(rig, "ia_cwf", route, stream, prompt) + assert status == 200, text + assert _FALLBACK_TEXT in text, text + assert _calls_with(rig.peers["ia_cwf"].drain(), prompt) == 1 + assert _calls_with(rig.peers["fallback"].drain(), prompt) == 1 + + +def test_generic_fallbacks_still_answer_an_overflow_from_a_provider_without_its_own_mapper(rig: Rig) -> None: + prompt: Final = _prompt("mimo fb") + status, text = _send(rig, "mimo_fb", "/v1/chat/completions", False, prompt) + assert status == 200, text + assert _FALLBACK_TEXT in text, text + assert _calls_with(rig.peers["mimo_fb"].drain(), prompt) == 1 + assert _calls_with(rig.peers["fallback"].drain(), prompt) == 1 + + +def test_a_plain_400_is_returned_without_the_context_window_fallback(rig: Rig) -> None: + prompt: Final = _prompt("ia plain") + status, text = _send(rig, "ia_plain", "/v1/chat/completions", False, prompt) + assert status == 400, text + assert _PLAIN_TEXT in text, text + assert "ContextWindowExceededError" not in text, text + assert _calls_with(rig.peers["ia_plain"].drain(), prompt) == 1 + assert _calls_with(rig.peers["fallback"].drain(), prompt) == 0 + + +def test_a_concurrent_overflow_burst_is_answered_once_per_request_by_the_fallback(rig: Rig) -> None: + plan: Final = tuple((*_SURFACES[index % len(_SURFACES)], _prompt(f"burst {index}")) for index in range(30)) + with ThreadPoolExecutor(max_workers=15) as pool: + results: Final = tuple(pool.map(partial(_send, rig, "ia_cwf"), *zip(*plan, strict=True))) + assert [status for status, _ in results] == [200] * len(plan), results + assert all(_FALLBACK_TEXT in text for _, text in results), results + overflow_calls: Final = rig.peers["ia_cwf"].drain() + fallback_calls: Final = rig.peers["fallback"].drain() + assert [_calls_with(overflow_calls, prompt) for _, _, prompt in plan] == [1] * len(plan) + assert [_calls_with(fallback_calls, prompt) for _, _, prompt in plan] == [1] * len(plan) + + +def _completion_id(status: int, text: str) -> str | None: + if status != 200: + return None + identity: Final = _JSON_OBJECT.validate_json(text)["id"] + assert isinstance(identity, str) + return identity + + +def _post_chat(rig: Rig, label: str) -> tuple[int, str]: + try: + return _send(rig, label, "/v1/chat/completions", False, _prompt(label)) + except httpx.TransportError as error: + return 0, repr(error) + + +def _spend_rows(identities: tuple[str, ...]) -> list[dict[str, JsonValue]]: + return read_rows( + 'SELECT request_id FROM "LiteLLM_SpendLogs" WHERE request_id = ANY(string_to_array(%s, \',\'))', + (",".join(identities),), + ) + + +def _assert_one_spend_row_each(identities: tuple[str, ...]) -> None: + rows: Final = eventually( + lambda: _spend_rows(identities), + lambda values: len(values) >= len(identities), + seconds=90, + ) + assert sorted(str(row["request_id"]) for row in rows) == sorted(identities) + + +def test_a_slow_vendor_under_a_burst_answers_every_request_once(rig: Rig) -> None: + with ThreadPoolExecutor(max_workers=16) as pool: + results: Final = tuple(pool.map(partial(_post_chat, rig), ("ia_slow",) * 16)) + assert [status for status, _ in results] == [200] * 16, results + identities: Final = tuple(_completion_id(status, text) or "" for status, text in results) + assert len(set(identities)) == 16 + assert len(rig.peers["ia_slow"].drain()) == 16 + _assert_one_spend_row_each(identities) + + +def test_killing_one_worker_mid_burst_leaves_the_sibling_serving_and_logging_once(rig: Rig) -> None: + workers: Final = eventually(rig.owned.worker_pids, lambda pids: len(pids) == 2, seconds=30) + with ThreadPoolExecutor(max_workers=12) as pool, rig.held.released_on_exit(): + in_flight: Final = tuple(pool.submit(_post_chat, rig, "ia_held") for _ in range(12)) + eventually(rig.held.held.qsize, lambda size: size == len(in_flight), seconds=120) + held_by: Final = MappingProxyType({pid: open_connections_to(pid, rig.peers["ia_held"].url) for pid in workers}) + assert sum(held_by.values()) == len(in_flight), held_by + victim_pid, survivor_pid = sorted(workers, key=held_by.__getitem__) + assert held_by[survivor_pid] >= 1, held_by + with rig.owned.replacing_worker(victim_pid) as kill: + kill() + rig.held.release.set() + during: Final = tuple(future.result() for future in in_flight) + after: Final = tuple(_post_chat(rig, "ia_ok") for _ in range(6)) + assert [status for status, _ in after] == [200] * 6, after + assert [status for status, _ in during].count(200) == held_by[survivor_pid], (held_by, during) + assert all(status in (200, 0, 500, 502, 503) for status, _ in during), during + identities: Final = tuple( + identity for identity in (_completion_id(status, text) for status, text in (*during, *after)) if identity + ) + assert len(set(identities)) == len(identities) + _assert_one_spend_row_each(identities) + assert rig.gateway.request("GET", "/health/liveliness").status_code == 200 diff --git a/tests/integration/providers/test_inferenceapis_wire.py b/tests/integration/providers/test_inferenceapis_wire.py new file mode 100644 index 00000000000..8288c59ad5d --- /dev/null +++ b/tests/integration/providers/test_inferenceapis_wire.py @@ -0,0 +1,580 @@ +import json +import uuid +from collections.abc import Callable, Mapping +from pathlib import Path +from typing import Final + +import anthropic +import httpx +import pytest +from integration._support.client import Gateway, eventually, object_value +from integration._support.database import read_rows +from integration._support.wire import Reply, Request, Wire, wire_server +from openai import OpenAI +from pydantic import JsonValue, TypeAdapter + +_BACKEND: Final = "deepseek-ai/DeepSeek-V4-Flash" +_MODEL: Final = f"inferenceapis/{_BACKEND}" +_API_KEY: Final = "synthetic-inferenceapis-key" +_OPENAI_KEY: Final = "synthetic-openai-key" +_OVERFLOW_TEXT: Final = "Input too long: 150055 input tokens, limit is 131072 for this model" +_PARAM_ERROR_TEXT: Final = ( + "Input should be 'error' or 'truncate', field: 'context_length_exceeded_behavior', value: 'foo'" +) +_WEATHER_TOOL: Final[JsonValue] = { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather for a city", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, + }, +} +_COST_MAP_PATH: Final = Path(__file__).resolve().parents[3] / "model_prices_and_context_window.json" +_COST_MAP: Final = TypeAdapter(dict[str, dict[str, JsonValue]]) +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_JSON_LIST: Final = TypeAdapter(list[JsonValue]) + + +def _identity(label: str) -> str: + return f"inferenceapis-{label}-{uuid.uuid4().hex}" + + +def _catalog_cost(field: str) -> float: + value: Final = _COST_MAP.validate_json(_COST_MAP_PATH.read_bytes())[_MODEL][field] + assert isinstance(value, (int, float)) + return float(value) + + +def _approx(value: float) -> object: + return pytest.approx(value, rel=1e-6) # pyright: ignore[reportUnknownMemberType] # pytest lacks typed approx stubs + + +def _sse(events: tuple[Mapping[str, JsonValue], ...], named: bool) -> tuple[bytes, ...]: + return tuple( + (f"event: {event['type']}\n" if named else "").encode() + f"data: {json.dumps(event)}\n\n".encode() + for event in events + ) + + +def _chat_completion(identity: str, message: Mapping[str, JsonValue], finish: str = "stop") -> Reply: + return Reply( + body=json.dumps( + { + "id": identity, + "object": "chat.completion", + "created": 1, + "model": _BACKEND, + "choices": [{"index": 0, "message": message, "finish_reason": finish}], + "usage": {"prompt_tokens": 23, "completion_tokens": 41, "total_tokens": 64}, + } + ).encode() + ) + + +def _chat_frames(identity: str, pieces: tuple[str, ...]) -> Reply: + chunk: Final[dict[str, JsonValue]] = { + "id": identity, + "object": "chat.completion.chunk", + "created": 1, + "model": _BACKEND, + } + deltas: Final[tuple[dict[str, JsonValue], ...]] = tuple( + {**chunk, "choices": [{"index": 0, "delta": {"content": piece}, "finish_reason": None}]} for piece in pieces + ) + finish: Final[dict[str, JsonValue]] = {**chunk, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]} + return Reply( + content_type="text/event-stream", + chunks=(*_sse((*deltas, finish), named=False), b"data: [DONE]\n\n"), + ) + + +def _response_object(identity: str, text: str, status: str) -> dict[str, JsonValue]: + return { + "id": f"resp_{identity}", + "object": "response", + "created_at": 1, + "status": status, + "model": _BACKEND, + "output": [ + { + "type": "message", + "id": f"msg_{identity}", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": text, "annotations": []}], + } + ] + if status == "completed" + else [], + "usage": {"input_tokens": 9, "output_tokens": 2, "total_tokens": 11}, + } + + +def _responses_reply(identity: str, text: str, stream: bool) -> Reply: + completed: Final = _response_object(identity, text, "completed") + if not stream: + return Reply(body=json.dumps(completed).encode()) + events: Final[tuple[Mapping[str, JsonValue], ...]] = ( + {"type": "response.created", "sequence_number": 0, "response": _response_object(identity, text, "in_progress")}, + { + "type": "response.output_text.delta", + "sequence_number": 1, + "item_id": f"msg_{identity}", + "output_index": 0, + "content_index": 0, + "delta": text, + }, + {"type": "response.completed", "sequence_number": 2, "response": completed}, + ) + return Reply(content_type="text/event-stream", chunks=_sse(events, named=True)) + + +def _message(identity: str, text: str) -> dict[str, JsonValue]: + return { + "id": identity, + "type": "message", + "role": "assistant", + "model": _BACKEND, + "content": [{"type": "text", "text": text}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 11, "output_tokens": 3}, + } + + +def _messages_reply(identity: str, text: str, stream: bool) -> Reply: + if not stream: + return Reply(body=json.dumps(_message(identity, text)).encode()) + events: Final[tuple[Mapping[str, JsonValue], ...]] = ( + {"type": "message_start", "message": {**_message(identity, ""), "content": [], "stop_reason": None}}, + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": text}}, + {"type": "content_block_stop", "index": 0}, + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": {"output_tokens": 3}, + }, + {"type": "message_stop"}, + ) + return Reply(content_type="text/event-stream", chunks=_sse(events, named=True)) + + +def _error(status: int, message: str, code: str | None, kind: str = "invalid_request_error") -> Reply: + return Reply(status=status, body=json.dumps({"error": {"message": message, "type": kind, "code": code}}).encode()) + + +def _body(request: Request) -> dict[str, JsonValue]: + return _JSON_OBJECT.validate_json(request.body) + + +def _assert_vendor_request(request: Request, target: str) -> dict[str, JsonValue]: + assert (request.method, request.target) == ("POST", target), request.target + assert request.headers["authorization"] == f"Bearer {_API_KEY}" + body: Final = _body(request) + assert body["model"] == _BACKEND + return body + + +def _chat(model: str, prompt: str, **extra: JsonValue) -> dict[str, JsonValue]: + return {"model": model, "messages": [{"role": "user", "content": prompt}], **extra} + + +def _messages(model: str, prompt: str, **extra: JsonValue) -> dict[str, JsonValue]: + return {"model": model, "max_tokens": 32, "messages": [{"role": "user", "content": prompt}], **extra} + + +def _responses(model: str, prompt: str, **extra: JsonValue) -> dict[str, JsonValue]: + return {"model": model, "input": prompt, **extra} + + +def _consumed(gateway: Gateway, path: str, body: Mapping[str, JsonValue]) -> tuple[httpx.Response, str]: + with gateway.client.stream("POST", path, json=body, headers={"Authorization": f"Bearer {gateway.key}"}) as response: + text: Final = b"".join(response.iter_bytes()).decode() + return response, text + + +def _data_events(text: str) -> tuple[dict[str, JsonValue], ...]: + return tuple( + _JSON_OBJECT.validate_json(line.removeprefix("data: ")) + for line in text.splitlines() + if line.startswith("data: ") and line != "data: [DONE]" + ) + + +def _targets(wire: Wire) -> list[tuple[str, str]]: + return [(request.method, request.target) for request in wire.drain()] + + +def test_chat_completion_reaches_the_vendor_and_bills_the_catalog_price(gateway: Gateway) -> None: + identity: Final = _identity("chat") + prompt: Final = f"chat {identity}" + + def respond(request: Request) -> Reply: + body: Final = _assert_vendor_request(request, "/v1/chat/completions") + assert body["messages"] == [{"role": "user", "content": prompt}] + return _chat_completion(identity, {"role": "assistant", "content": "43"}) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=_MODEL, api_base=f"{wire.url}/v1", api_key=_API_KEY) + response: Final = gateway.request("POST", "/v1/chat/completions", _chat(model, prompt)) + assert response.status_code == 200, response.text + payload: Final = _JSON_OBJECT.validate_json(response.content) + assert payload["id"] == identity + choice: Final = object_value(_JSON_LIST.validate_python(payload["choices"])[0]) + assert object_value(choice["message"])["content"] == "43" + expected_cost: Final = 23 * _catalog_cost("input_cost_per_token") + 41 * _catalog_cost("output_cost_per_token") + assert float(response.headers["x-litellm-response-cost"]) == _approx(expected_cost) + assert _targets(wire) == [("POST", "/v1/chat/completions")] + rows: Final = eventually( + lambda: read_rows( + 'SELECT spend, prompt_tokens, completion_tokens FROM "LiteLLM_SpendLogs" WHERE request_id=%s', + (identity,), + ), + lambda values: len(values) == 1, + seconds=70, + ) + assert (rows[0]["prompt_tokens"], rows[0]["completion_tokens"]) == (23, 41) + spend: Final = rows[0]["spend"] + assert isinstance(spend, (int, float, str)) + assert float(spend) == _approx(expected_cost) + + +def test_streamed_chat_completion_delivers_every_delta_and_one_done(gateway: Gateway) -> None: + identity: Final = _identity("chat-stream") + pieces: Final = ("1\n2\n", "3\n4\n", "5") + + def respond(request: Request) -> Reply: + assert _assert_vendor_request(request, "/v1/chat/completions")["stream"] is True + return _chat_frames(identity, pieces) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=_MODEL, api_base=f"{wire.url}/v1", api_key=_API_KEY) + response, text = _consumed(gateway, "/v1/chat/completions", _chat(model, f"count {identity}", stream=True)) + assert response.status_code == 200, text + assert text.count("data: [DONE]") == 1 + events: Final = _data_events(text) + assert {str(event["id"]) for event in events} == {identity} + deltas: Final = tuple( + object_value(object_value(choice)["delta"]).get("content") or "" + for event in events + for choice in _JSON_LIST.validate_python(event["choices"]) + ) + assert "".join(str(delta) for delta in deltas) == "".join(pieces) + assert _targets(wire) == [("POST", "/v1/chat/completions")] + + +@pytest.mark.parametrize("stream", (False, True), ids=("json", "stream")) +def test_responses_call_goes_to_the_vendor_responses_endpoint(gateway: Gateway, stream: bool) -> None: + identity: Final = _identity("responses") + answer: Final = f"answer {identity}" + prompt: Final = f"responses {identity}" + + def respond(request: Request) -> Reply: + body: Final = _assert_vendor_request(request, "/v1/responses") + assert body["input"] == prompt + assert bool(body.get("stream")) is stream + return _responses_reply(identity, answer, stream) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=_MODEL, api_base=f"{wire.url}/v1", api_key=_API_KEY) + response, text = _consumed(gateway, "/v1/responses", _responses(model, prompt, stream=stream)) + assert response.status_code == 200, text + completed: Final = ( + _JSON_OBJECT.validate_json(text) + if not stream + else object_value( + next(event for event in _data_events(text) if event["type"] == "response.completed")["response"] + ) + ) + assert completed["status"] == "completed" + assert answer in json.dumps(completed["output"]) + assert _targets(wire) == [("POST", "/v1/responses")] + + +@pytest.mark.parametrize("stream", (False, True), ids=("json", "stream")) +def test_messages_call_goes_to_the_vendor_messages_endpoint_untranslated(gateway: Gateway, stream: bool) -> None: + identity: Final = _identity("messages") + answer: Final = f"answer {identity}" + prompt: Final = f"messages {identity}" + + def respond(request: Request) -> Reply: + body: Final = _assert_vendor_request(request, "/v1/messages") + assert body["messages"] == [{"role": "user", "content": prompt}] + assert body["max_tokens"] == 32 + assert "anthropic-version" in request.headers + assert bool(body.get("stream")) is stream + return _messages_reply(identity, answer, stream) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=_MODEL, api_base=f"{wire.url}/v1", api_key=_API_KEY) + response, text = _consumed(gateway, "/v1/messages", _messages(model, prompt, stream=stream)) + assert response.status_code == 200, text + if stream: + events: Final = _data_events(text) + assert [event["type"] for event in events][-1] == "message_stop" + assert object_value(events[0]["message"])["id"] == identity + assert answer in text + else: + payload: Final = _JSON_OBJECT.validate_json(text) + assert payload["id"] == identity + assert payload["content"] == [{"type": "text", "text": answer}] + assert _targets(wire) == [("POST", "/v1/messages")] + + +def test_chat_tool_call_round_trips(gateway: Gateway) -> None: + identity: Final = _identity("tool") + tool_call: Final[JsonValue] = { + "id": "call_paris", + "type": "function", + "function": {"name": "get_weather", "arguments": json.dumps({"city": "Paris"})}, + } + + def respond(request: Request) -> Reply: + body: Final = _assert_vendor_request(request, "/v1/chat/completions") + assert body["tools"] == [_WEATHER_TOOL] + assert body["tool_choice"] == "auto" + message: Final[dict[str, JsonValue]] = {"role": "assistant", "content": None, "tool_calls": [tool_call]} + return _chat_completion(identity, message, "tool_calls") + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=_MODEL, api_base=f"{wire.url}/v1", api_key=_API_KEY) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + _chat(model, "weather in Paris", tools=[_WEATHER_TOOL], tool_choice="auto"), + ) + assert response.status_code == 200, response.text + payload: Final = _JSON_OBJECT.validate_json(response.content) + assert payload["id"] == identity + choice: Final = object_value(_JSON_LIST.validate_python(payload["choices"])[0]) + assert choice["finish_reason"] == "tool_calls" + assert object_value(choice["message"])["tool_calls"] == [tool_call] + assert _targets(wire) == [("POST", "/v1/chat/completions")] + + +def test_non_openai_params_reach_the_vendor_body_top_level(gateway: Gateway) -> None: + identity: Final = _identity("extra") + + def respond(request: Request) -> Reply: + body: Final = _assert_vendor_request(request, "/v1/chat/completions") + assert body["top_k"] == 5 + assert body["chat_template_kwargs"] == {"enable_thinking": False} + assert "extra_body" not in body + return _chat_completion(identity, {"role": "assistant", "content": "ok"}) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=_MODEL, api_base=f"{wire.url}/v1", api_key=_API_KEY) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + _chat(model, "extra params", top_k=5, chat_template_kwargs={"enable_thinking": False}), + ) + assert response.status_code == 200, response.text + assert _JSON_OBJECT.validate_json(response.content)["id"] == identity + assert _targets(wire) == [("POST", "/v1/chat/completions")] + + +_OVERFLOW_CASES: Final = ( + ("/v1/chat/completions", "/v1/chat/completions", "context_length_exceeded"), + ("/v1/chat/completions", "/v1/chat/completions", "upstream_rejected"), + ("/v1/messages", "/v1/messages", "upstream_rejected"), + ("/v1/responses", "/v1/responses", "upstream_rejected"), +) +_REQUEST_BUILDERS: Final[Mapping[str, Callable[[str, str], dict[str, JsonValue]]]] = { + "/v1/chat/completions": _chat, + "/v1/messages": _messages, + "/v1/responses": _responses, +} + + +@pytest.mark.parametrize(("route", "target", "code"), _OVERFLOW_CASES) +def test_vendor_overflow_reaches_the_caller_as_a_context_window_error( + gateway: Gateway, route: str, target: str, code: str +) -> None: + prompt: Final = f"overflow {uuid.uuid4().hex}" + + def respond(request: Request) -> Reply: + _assert_vendor_request(request, target) + return _error(400, _OVERFLOW_TEXT, code) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=_MODEL, api_base=f"{wire.url}/v1", api_key=_API_KEY) + response: Final = gateway.request("POST", route, _REQUEST_BUILDERS[route](model, prompt)) + assert response.status_code == 400, response.text + assert "ContextWindowExceededError" in response.text, response.text + assert _OVERFLOW_TEXT in response.text, response.text + assert _targets(wire) == [("POST", target)] + + +@pytest.mark.parametrize( + ("status", "message", "code", "kind", "caller_status", "caller_class"), + ( + (400, _PARAM_ERROR_TEXT, None, "invalid_request_error", 400, "BadRequestError"), + (401, "Invalid API key", "invalid_api_key", "authentication_error", 401, "AuthenticationError"), + ), + ids=("param-error-naming-the-overflow-field", "bad-key"), +) +def test_vendor_errors_that_are_not_overflow_keep_their_class( + gateway: Gateway, status: int, message: str, code: str | None, kind: str, caller_status: int, caller_class: str +) -> None: + def respond(request: Request) -> Reply: + _assert_vendor_request(request, "/v1/chat/completions") + return _error(status, message, code, kind) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=_MODEL, api_base=f"{wire.url}/v1", api_key=_API_KEY) + response: Final = gateway.request("POST", "/v1/chat/completions", _chat(model, f"error {uuid.uuid4().hex}")) + assert response.status_code == caller_status, response.text + assert caller_class in response.text, response.text + assert "ContextWindowExceededError" not in response.text, response.text + assert _targets(wire) == [("POST", "/v1/chat/completions")] + + +def _echoing(_: Request) -> Reply: + return _chat_completion(_identity("echo"), {"role": "assistant", "content": "ok"}) + + +def _forwarded_top_k(wire: Wire) -> list[JsonValue]: + return [_body(request).get("top_k") for request in wire.drain()] + + +@pytest.mark.parametrize( + "value", + (5, [1, 2], "", "x" * 5120), + ids=("int", "list", "empty-string", "five-kb"), +) +def test_a_non_openai_param_of_any_shape_forwards_like_it_does_for_an_openai_deployment( + gateway: Gateway, value: JsonValue +) -> None: + with ( + wire_server(_echoing) as vendor, + wire_server(_echoing) as openai_peer, + gateway.scenario() as scenario, + ): + model: Final = scenario.model(model=_MODEL, api_base=f"{vendor.url}/v1", api_key=_API_KEY) + control: Final = scenario.model( + model="openai/gpt-4o-mini", api_base=f"{openai_peer.url}/v1", api_key=_OPENAI_KEY + ) + responses: Final = tuple( + gateway.request("POST", "/v1/chat/completions", _chat(name, "shape", top_k=value)) + for name in (model, control) + ) + assert [response.status_code for response in responses] == [200, 200], [r.text for r in responses] + assert _forwarded_top_k(vendor) == [value] + assert _forwarded_top_k(openai_peer) == [value] + + +def test_a_param_sent_twice_forwards_the_last_value_like_an_openai_deployment(gateway: Gateway) -> None: + with ( + wire_server(_echoing) as vendor, + wire_server(_echoing) as openai_peer, + gateway.scenario() as scenario, + ): + model: Final = scenario.model(model=_MODEL, api_base=f"{vendor.url}/v1", api_key=_API_KEY) + control: Final = scenario.model( + model="openai/gpt-4o-mini", api_base=f"{openai_peer.url}/v1", api_key=_OPENAI_KEY + ) + responses: Final = tuple( + gateway.client.post( + "/v1/chat/completions", + content=( + f'{{"model": "{name}", "messages": [{{"role": "user", "content": "twice"}}], ' + '"top_k": 1, "top_k": 2}' + ).encode(), + headers={"Authorization": f"Bearer {gateway.key}", "Content-Type": "application/json"}, + ) + for name in (model, control) + ) + assert [response.status_code for response in responses] == [200, 200], [r.text for r in responses] + assert _forwarded_top_k(vendor) == [2] + assert _forwarded_top_k(openai_peer) == [2] + + +def test_messages_on_a_json_provider_without_native_messages_forwards_top_k(gateway: Gateway) -> None: + identity: Final = _identity("bridge") + + def respond(request: Request) -> Reply: + assert (request.method, request.target) == ("POST", "/v1/chat/completions"), request.target + assert _body(request)["top_k"] == 5 + return _chat_completion(identity, {"role": "assistant", "content": "bridged"}) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model="veniceai/venice-uncensored", api_base=f"{wire.url}/v1", api_key=_API_KEY) + response: Final = gateway.request("POST", "/v1/messages", _messages(model, "bridge", top_k=5)) + assert response.status_code == 200, response.text + assert _JSON_OBJECT.validate_json(response.content)["content"] == [{"type": "text", "text": "bridged"}] + assert _targets(wire) == [("POST", "/v1/chat/completions")] + + +def _lists_inferenceapis(endpoint: Mapping[str, JsonValue]) -> bool: + providers: Final = _JSON_LIST.validate_python(endpoint["providers"]) + return any(object_value(provider)["slug"] == "inferenceapis" for provider in providers) + + +def test_public_routes_list_the_provider_with_its_endpoints_and_fields(gateway: Gateway) -> None: + providers: Final = gateway.request("GET", "/public/providers") + assert providers.status_code == 200, providers.text + assert "inferenceapis" in _JSON_LIST.validate_json(providers.content) + + fields_response: Final = gateway.request("GET", "/public/providers/fields") + assert fields_response.status_code == 200, fields_response.text + entries: Final = tuple( + object_value(entry) + for entry in _JSON_LIST.validate_json(fields_response.content) + if object_value(entry)["litellm_provider"] == "inferenceapis" + ) + assert len(entries) == 1 + required: Final = { + object_value(field)["key"]: object_value(field)["required"] + for field in _JSON_LIST.validate_python(entries[0]["credential_fields"]) + } + assert required == {"api_base": False, "api_key": True} + + endpoints_response: Final = gateway.request("GET", "/public/endpoints") + assert endpoints_response.status_code == 200, endpoints_response.text + endpoints: Final = tuple( + object_value(endpoint) + for endpoint in _JSON_LIST.validate_python(_JSON_OBJECT.validate_json(endpoints_response.content)["endpoints"]) + ) + listed: Final = frozenset(str(endpoint["key"]) for endpoint in endpoints if _lists_inferenceapis(endpoint)) + assert listed == {"chat_completions", "messages", "responses"} + + +def test_openai_and_anthropic_sdk_clients_reach_the_vendor(gateway: Gateway) -> None: + chat_identity: Final = _identity("sdk-chat") + message_identity: Final = _identity("sdk-messages") + + def respond(request: Request) -> Reply: + if request.target == "/v1/messages": + _assert_vendor_request(request, "/v1/messages") + return _messages_reply(message_identity, "from messages", stream=False) + _assert_vendor_request(request, "/v1/chat/completions") + return _chat_completion(chat_identity, {"role": "assistant", "content": "from chat"}) + + base_url: Final = str(gateway.client.base_url).rstrip("/") + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=_MODEL, api_base=f"{wire.url}/v1", api_key=_API_KEY) + openai_client: Final = OpenAI( + api_key=gateway.key, + base_url=f"{base_url}/v1", + max_retries=0, + http_client=httpx.Client(timeout=30, trust_env=False), + ) + completion: Final = openai_client.chat.completions.create( + model=model, messages=[{"role": "user", "content": "sdk chat"}] + ) + assert completion.id == chat_identity + assert completion.choices[0].message.content == "from chat" + anthropic_client: Final = anthropic.Anthropic( + base_url=base_url, api_key=gateway.key, auth_token=gateway.key, max_retries=0 + ) + message: Final = anthropic_client.messages.create( + model=model, max_tokens=32, messages=[{"role": "user", "content": "sdk messages"}] + ) + assert message.id == message_identity + assert len(message.content) == 1 + block: Final = message.content[0] + assert isinstance(block, anthropic.types.TextBlock) + assert block.text == "from messages" + assert sorted(_targets(wire)) == [("POST", "/v1/chat/completions"), ("POST", "/v1/messages")] diff --git a/tests/integration/sdk/test_provider_error_and_extra_body_families_wire.py b/tests/integration/sdk/test_provider_error_and_extra_body_families_wire.py new file mode 100644 index 00000000000..dc991e25730 --- /dev/null +++ b/tests/integration/sdk/test_provider_error_and_extra_body_families_wire.py @@ -0,0 +1,304 @@ +from __future__ import annotations + +import asyncio +import json +from collections.abc import Callable, Mapping +from typing import Final + +import pytest +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import rsa +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +import litellm +import openai +from litellm import Router + +_KEY: Final = "synthetic-family-key" +_PROMPT: Final = "family wire probe" +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_OVERFLOW_TEXT: Final = "This model's maximum context length is 4096 tokens. However, you requested 9000 tokens." +_INFERENCEAPIS_TEXT: Final = "Input too long: 150055 input tokens, limit is 131072 for this model" +_PLAIN_TEXT: Final = "Invalid value for 'temperature': expected a number between 0 and 2." +_FIREWORKS_PARAM_TEXT: Final = ( + "Input should be 'error' or 'truncate', field: 'context_length_exceeded_behavior', value: 'foo'" +) +_TOKEN_REPLY: Final = Reply( + body=json.dumps( + {"access_token": "synthetic-token", "expires_at": 4102444800000, "expires_in": 3600, "token_type": "bearer"} + ).encode() +) +_STATUS_MAPPED_JSON_PROVIDERS: Final = ( + "veniceai", + "xiaomi_mimo", + "scaleway", + "abliteration", + "llamagate", + "gmi", + "sarvam", + "assemblyai", + "charity_engine", + "aihubmix", + "crusoe", + "neosantara", + "empiriolabs", +) +_EXTRA_BODY_MEMBERS: Final = (*_STATUS_MAPPED_JSON_PROVIDERS, "inferenceapis") +_OPENAI_COMPATIBLE_JSON_MEMBER: Final = "nano-gpt" +_PLAIN_BUILDERS: Final = ( + *_STATUS_MAPPED_JSON_PROVIDERS, + "a2a", + "amazon_nova", + "cloudflare", + "compactifai", + "custom", + "datarobot", + "heroku", + "langflow", + "langgraph", + "maritalk", + "minimax", + "nadir", + "ollama", + "ollama_chat", + "oobabooga", + "ovhcloud", + "petals", + "snowflake", + "text-completion-codestral", + "text-completion-inception", +) +_HANDLER_BRANCHES: Final = ("openai", "anthropic", "deepinfra", "gemini") + + +def _oci_key() -> str: + key: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048) + return key.private_bytes( + serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption() + ).decode() + + +def _plain_call(provider: str) -> Callable[[str], dict[str, JsonValue]]: + return lambda url: {"model": f"{provider}/m", "api_base": url, "api_key": _KEY} + + +_SPECIAL_BUILDERS: Final[Mapping[str, Callable[[str], dict[str, JsonValue]]]] = { + "triton": lambda url: {"model": "triton/m", "api_base": f"{url}/v2/models/m/generate"}, + "watsonx_text": lambda url: {"model": "watsonx_text/m", "api_base": url, "token": "t", "project_id": "p"}, + "gigachat": lambda url: { + "model": "gigachat/m", + "api_base": url, + "api_key": _KEY, + "gigachat_auth_url": f"{url}/oauth", + }, + "scaledown": lambda url: {"model": "scaledown/summarize", "api_base": url, "api_key": _KEY}, + "gradient_ai": lambda url: {"model": "gradient_ai/m", "api_base": url, "api_key": _KEY, "drop_params": True}, + "oci": lambda url: { + "model": "oci/meta.llama-3.3-70b-instruct", + "api_base": url, + "oci_key": _oci_key(), + "oci_user": "ocid1.user.oc1..synthetic", + "oci_fingerprint": "aa:bb:cc", + "oci_tenancy": "ocid1.tenancy.oc1..synthetic", + "oci_region": "us-chicago-1", + "oci_compartment_id": "ocid1.compartment.oc1..synthetic", + }, +} +_BUILDER_CALLS: Final[Mapping[str, Callable[[str], dict[str, JsonValue]]]] = { + **{provider: _plain_call(provider) for provider in _PLAIN_BUILDERS}, + **_SPECIAL_BUILDERS, +} + + +def _error(status: int, message: str, code: str | None = None) -> Reply: + error: Final[dict[str, JsonValue]] = {"message": message, "type": "invalid_request_error", "code": code} + return Reply(status=status, body=json.dumps({"error": error}).encode()) + + +def _is_token_request(request: Request) -> bool: + return "oauth" in request.target or "token" in request.target + + +def _scripted(reply: Reply) -> Callable[[Request], Reply]: + return lambda request: _TOKEN_REPLY if _is_token_request(request) else reply + + +def _prompted(wire: Wire) -> int: + return sum(1 for request in wire.drain() if not _is_token_request(request)) + + +def _completion(model: str) -> Reply: + return Reply( + body=json.dumps( + { + "id": "chatcmpl-family", + "object": "chat.completion", + "created": 1, + "model": model, + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4}, + } + ).encode() + ) + + +def _answer(response: object) -> str | None: + assert isinstance(response, litellm.ModelResponse) + choice: Final = response.choices[0] + assert isinstance(choice, litellm.Choices) + return choice.message.content + + +async def _raised(call: dict[str, JsonValue]) -> BaseException: + with pytest.raises(openai.APIError) as caught: + await litellm.acompletion(messages=[{"role": "user", "content": _PROMPT}], num_retries=0, **call) + return caught.value + + +@pytest.mark.parametrize( + ("text", "code"), + ((_OVERFLOW_TEXT, "context_length_exceeded"), (_INFERENCEAPIS_TEXT, "upstream_rejected")), + ids=("standard-phrase", "inference-apis-phrase"), +) +def test_an_overflow_400_from_inference_apis_raises_context_window_exceeded(text: str, code: str) -> None: + with wire_server(_scripted(_error(400, text, code))) as wire: + raised: Final = asyncio.run(_raised(_plain_call("inferenceapis")(wire.url))) + assert isinstance(raised, litellm.ContextWindowExceededError), repr(raised) + assert raised.status_code == 400 + assert text in str(raised) + assert _prompted(wire) == 1 + + +@pytest.mark.parametrize("text", (_OVERFLOW_TEXT, _PLAIN_TEXT), ids=("overflow", "plain")) +@pytest.mark.parametrize("provider", sorted(_BUILDER_CALLS)) +def test_a_400_stays_a_bad_request_for_every_provider_without_its_own_mapper(provider: str, text: str) -> None: + with wire_server(_scripted(_error(400, text))) as wire: + raised: Final = asyncio.run(_raised(_BUILDER_CALLS[provider](wire.url))) + assert isinstance(raised, litellm.BadRequestError), repr(raised) + assert not isinstance(raised, litellm.ContextWindowExceededError), repr(raised) + assert text in str(raised) + assert _prompted(wire) == 1 + + +@pytest.mark.parametrize("provider", _HANDLER_BRANCHES) +def test_the_inference_apis_overflow_phrase_raises_context_window_exceeded_on_providers_with_a_mapper( + provider: str, +) -> None: + with wire_server(_scripted(_error(400, _INFERENCEAPIS_TEXT, "upstream_rejected"))) as wire: + raised: Final = asyncio.run(_raised(_plain_call(provider)(wire.url))) + assert isinstance(raised, litellm.ContextWindowExceededError), repr(raised) + assert _INFERENCEAPIS_TEXT in str(raised) + assert _prompted(wire) == 1 + + +def test_a_param_error_naming_context_length_exceeded_behavior_stays_a_bad_request() -> None: + with wire_server(_scripted(_error(400, _FIREWORKS_PARAM_TEXT))) as wire: + call: Final[dict[str, JsonValue]] = { + "model": "fireworks_ai/accounts/fireworks/models/m", + "api_base": wire.url, + "api_key": _KEY, + } + raised: Final = asyncio.run(_raised(call)) + assert isinstance(raised, litellm.BadRequestError), repr(raised) + assert not isinstance(raised, litellm.ContextWindowExceededError), repr(raised) + assert _prompted(wire) == 1 + + +def test_aiohttp_openai_still_reports_an_upstream_400_as_a_server_error() -> None: + with wire_server(_scripted(_error(400, _OVERFLOW_TEXT, "context_length_exceeded"))) as wire: + raised: Final = asyncio.run(_raised(_plain_call("aiohttp_openai")(wire.url))) + assert isinstance(raised, litellm.InternalServerError), repr(raised) + assert _prompted(wire) == 1 + + +_FALLBACK_SETUPS: Final[Mapping[str, tuple[str, list[dict[str, list[str]]], list[dict[str, list[str]]]]]] = { + "context-window-fallbacks": ("inferenceapis/m", [{"primary": ["backup"]}], []), + "generic-fallbacks-beside-context-window-fallbacks": ( + "xiaomi_mimo/m", + [{"other": ["backup"]}], + [{"primary": ["backup"]}], + ), +} + + +def _fallback_router(setup: str, primary_url: str, backup_url: str) -> Router: + primary_model, context_window_fallbacks, fallbacks = _FALLBACK_SETUPS[setup] + return Router( + model_list=[ + { + "model_name": "primary", + "litellm_params": {"model": primary_model, "api_base": primary_url, "api_key": _KEY}, + }, + { + "model_name": "other", + "litellm_params": {"model": "openai/other", "api_base": backup_url, "api_key": _KEY}, + }, + { + "model_name": "backup", + "litellm_params": {"model": "openai/backup", "api_base": backup_url, "api_key": _KEY}, + }, + ], + context_window_fallbacks=context_window_fallbacks, + fallbacks=fallbacks, + num_retries=0, + ) + + +@pytest.mark.parametrize("client", ("sync", "async")) +@pytest.mark.parametrize("setup", sorted(_FALLBACK_SETUPS)) +def test_the_configured_fallback_answers_an_overflow(setup: str, client: str) -> None: + with ( + wire_server(_scripted(_error(400, _OVERFLOW_TEXT, "context_length_exceeded"))) as primary, + wire_server(lambda _: _completion("backup")) as backup, + ): + router: Final = _fallback_router(setup, primary.url, backup.url) + response: Final[object] = ( + router.completion(model="primary", messages=[{"role": "user", "content": _PROMPT}]) + if client == "sync" + else asyncio.run(router.acompletion(model="primary", messages=[{"role": "user", "content": _PROMPT}])) + ) + assert _answer(response) == "ok" + assert _prompted(primary) == 1 + assert _prompted(backup) == 1 + + +def _forwarding(provider_model: str) -> Callable[[Request], Reply]: + return lambda _: _completion(provider_model) + + +@pytest.mark.parametrize("provider", (*_EXTRA_BODY_MEMBERS, _OPENAI_COMPATIBLE_JSON_MEMBER)) +def test_every_json_provider_forwards_non_openai_params_in_the_request_body(provider: str) -> None: + with wire_server(_forwarding("m")) as wire: + response: Final = asyncio.run( + litellm.acompletion( + model=f"{provider}/m", + api_base=wire.url, + api_key=_KEY, + messages=[{"role": "user", "content": _PROMPT}], + top_k=5, + chat_template_kwargs={"enable_thinking": False}, + num_retries=0, + ) + ) + assert _answer(response) == "ok" + requests: Final = wire.drain() + assert len(requests) == 1 + body: Final = _JSON_OBJECT.validate_json(requests[0].body) + assert body["top_k"] == 5 + assert body["chat_template_kwargs"] == {"enable_thinking": False} + + +def test_a_sync_json_provider_call_forwards_non_openai_params_in_the_request_body() -> None: + with wire_server(_forwarding("m")) as wire: + litellm.completion( + model="veniceai/m", + api_base=wire.url, + api_key=_KEY, + messages=[{"role": "user", "content": _PROMPT}], + top_k=7, + num_retries=0, + ) + requests: Final = wire.drain() + assert len(requests) == 1 + assert _JSON_OBJECT.validate_json(requests[0].body)["top_k"] == 7 diff --git a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py index 06aa167d0ba..02ebae6dcf6 100644 --- a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py @@ -129,6 +129,14 @@ context_window_test_cases = [ "Invalid 'metadata': maximum input length is 512 characters.", False, ), + ( + '{"error":{"message":"Input too long: 150055 input tokens, limit is 131072 for this model","code":"upstream_rejected"}}', + True, + ), + ( + "Input should be 'error' or 'truncate', field: 'context_length_exceeded_behavior', value: 'foo'", + False, + ), # Negative cases (should return False) ("A generic API error occurred.", False), ("Invalid API Key provided.", False), @@ -830,6 +838,10 @@ DEVIATIONS_FROM_THE_OPENAI_SHAPE = { 422: (litellm.APIError, 422), 500: (litellm.APIError, 500), }, + "inferenceapis": { + 403: PERMISSION_DENIED, + 422: (litellm.BadRequestError, 400), + }, "mistral": {402: (litellm.APIError, 402)}, "nlp_cloud": { 402: (litellm.RateLimitError, 429), @@ -876,6 +888,7 @@ PROVIDERS_WITH_A_HANDLER = ( "gemini", "groq", "huggingface", + "inferenceapis", "mistral", "nlp_cloud", "ollama", @@ -1119,6 +1132,7 @@ PROVIDERS_THAT_RECOGNISE_A_FULL_CONTEXT_WINDOW = ( "fireworks_ai", "gemini", "groq", + "inferenceapis", "mistral", "openai", "perplexity", diff --git a/tests/unit/llms/openai_like/test_inferenceapis_provider.py b/tests/unit/llms/openai_like/test_inferenceapis_provider.py new file mode 100644 index 00000000000..29b2640bed7 --- /dev/null +++ b/tests/unit/llms/openai_like/test_inferenceapis_provider.py @@ -0,0 +1,272 @@ +import json +from typing import Final + +import httpx +import pytest +import respx + +import litellm +from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.llms.openai_like.json_loader import JSONProviderRegistry + +INFERENCEAPIS_API_BASE: Final = "https://api.inferenceapis.com/v1" +INFERENCEAPIS_MODEL_ID: Final = "deepseek-ai/DeepSeek-V4-Flash" +INFERENCEAPIS_MODEL: Final = f"inferenceapis/{INFERENCEAPIS_MODEL_ID}" + + +def _chat_completion_body(usage: dict[str, object]) -> dict[str, object]: + return { + "id": "chatcmpl_inferenceapis", + "object": "chat.completion", + "created": 1_789_550_000, + "model": INFERENCEAPIS_MODEL_ID, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Hello from Inference APIs"}, + "finish_reason": "stop", + } + ], + "usage": usage, + } + + +def test_inferenceapis_provider_resolution(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("INFERENCEAPIS_API_KEY", "inferenceapis-env-key") + + model, provider, api_key, api_base = get_llm_provider( + model=INFERENCEAPIS_MODEL, + custom_llm_provider=None, + api_base=None, + api_key=None, + ) + + assert model == INFERENCEAPIS_MODEL_ID + assert provider == "inferenceapis" + assert api_key == "inferenceapis-env-key" + assert api_base == INFERENCEAPIS_API_BASE + + +def test_inferenceapis_provider_keeps_explicit_credentials(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("INFERENCEAPIS_API_KEY", "inferenceapis-env-key") + + _, provider, api_key, api_base = get_llm_provider( + model="inferenceapis/openai/gpt-oss-120b", + custom_llm_provider=None, + api_base="https://inferenceapis.internal.example/v1", + api_key="inferenceapis-explicit-key", + ) + + assert provider == "inferenceapis" + assert api_key == "inferenceapis-explicit-key" + assert api_base == "https://inferenceapis.internal.example/v1" + + +def test_inferenceapis_cost_map_rows_only_list_registered_endpoints(): + provider: Final = JSONProviderRegistry.get("inferenceapis") + assert provider is not None + registered: Final = set(provider.supported_endpoints) + rows: Final = { + name: info for name, info in litellm.model_cost.items() if info.get("litellm_provider") == "inferenceapis" + } + + assert rows + assert all(set(info["supported_endpoints"]) <= registered for info in rows.values()) + + +def test_inferenceapis_chat_completion_request(): + with respx.mock() as upstream: + route: Final = upstream.post(f"{INFERENCEAPIS_API_BASE}/chat/completions").respond( + 200, json=_chat_completion_body({"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7}) + ) + response: Final = litellm.completion( + model=INFERENCEAPIS_MODEL, + messages=[{"role": "user", "content": "Say hello"}], + max_completion_tokens=7, + api_key="inferenceapis-test-key", + ) + + request: Final = route.calls.last.request + body: Final = json.loads(request.content) + assert route.call_count == 1 + assert request.headers["authorization"] == "Bearer inferenceapis-test-key" + assert body["model"] == INFERENCEAPIS_MODEL_ID + assert body["messages"] == [{"role": "user", "content": "Say hello"}] + assert body["max_completion_tokens"] == 7 + assert "max_tokens" not in body + assert response.choices[0].message.content == "Hello from Inference APIs" + + +def test_inferenceapis_chat_completion_reads_key_and_base_from_env(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("INFERENCEAPIS_API_KEY", "inferenceapis-env-key") + monkeypatch.setenv("INFERENCEAPIS_API_BASE", "https://inferenceapis.internal.example/v1") + with respx.mock() as upstream: + route: Final = upstream.post("https://inferenceapis.internal.example/v1/chat/completions").respond( + 200, json=_chat_completion_body({"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7}) + ) + litellm.completion(model=INFERENCEAPIS_MODEL, messages=[{"role": "user", "content": "Say hello"}]) + + assert route.call_count == 1 + assert route.calls.last.request.headers["authorization"] == "Bearer inferenceapis-env-key" + + +def test_inferenceapis_chat_completion_sends_provider_specific_params_in_the_body(): + with respx.mock() as upstream: + route: Final = upstream.post(f"{INFERENCEAPIS_API_BASE}/chat/completions").respond( + 200, json=_chat_completion_body({"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7}) + ) + response: Final = litellm.completion( + model=INFERENCEAPIS_MODEL, + messages=[{"role": "user", "content": "Say hello"}], + top_k=5, + chat_template_kwargs={"enable_thinking": False}, + api_key="inferenceapis-test-key", + ) + + body: Final = json.loads(route.calls.last.request.content) + assert body["top_k"] == 5 + assert body["chat_template_kwargs"] == {"enable_thinking": False} + assert response.choices[0].message.content == "Hello from Inference APIs" + + +def test_inferenceapis_context_length_error_raises_context_window_exceeded(): + with respx.mock() as upstream: + upstream.post(f"{INFERENCEAPIS_API_BASE}/chat/completions").respond( + 400, + json={ + "error": { + "message": "Input too long: 150055 input tokens, limit is 131072 for this model", + "type": "invalid_request_error", + "param": None, + "code": "context_length_exceeded", + } + }, + ) + with pytest.raises(litellm.ContextWindowExceededError): + litellm.completion( + model=INFERENCEAPIS_MODEL, + messages=[{"role": "user", "content": "Say hello"}], + api_key="inferenceapis-test-key", + ) + + +def test_inferenceapis_responses_request(): + with respx.mock() as upstream: + route: Final = upstream.post(f"{INFERENCEAPIS_API_BASE}/responses").respond( + 200, + json={ + "id": "resp_inferenceapis", + "object": "response", + "created_at": 1_789_550_000, + "model": INFERENCEAPIS_MODEL_ID, + "status": "completed", + "output": [ + { + "id": "msg_inferenceapis", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "Hello from Inference APIs", "annotations": []}], + } + ], + "usage": {"input_tokens": 4, "output_tokens": 3, "total_tokens": 7}, + }, + ) + response: Final = litellm.responses( + model=INFERENCEAPIS_MODEL, + input="Say hello", + api_key="inferenceapis-test-key", + ) + + request: Final = route.calls.last.request + body: Final = json.loads(request.content) + assert route.call_count == 1 + assert request.headers["authorization"] == "Bearer inferenceapis-test-key" + assert body["model"] == INFERENCEAPIS_MODEL_ID + assert body["input"] == "Say hello" + assert response.output[0].content[0].text == "Hello from Inference APIs" + + +@pytest.mark.asyncio +async def test_inferenceapis_anthropic_messages_request_is_sent_natively(): + with respx.mock() as upstream: + route: Final = upstream.post(f"{INFERENCEAPIS_API_BASE}/messages").respond( + 200, + json={ + "id": "msg_inferenceapis", + "type": "message", + "role": "assistant", + "model": INFERENCEAPIS_MODEL_ID, + "content": [{"type": "text", "text": "Hello from Inference APIs"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 4, "output_tokens": 3}, + }, + ) + response: Final = await litellm.anthropic.messages.acreate( + model=INFERENCEAPIS_MODEL, + messages=[{"role": "user", "content": "Say hello"}], + max_tokens=32, + api_key="inferenceapis-test-key", + client=AsyncHTTPHandler(transport=httpx.AsyncHTTPTransport()), + ) + + request: Final = route.calls.last.request + body: Final = json.loads(request.content) + assert route.call_count == 1 + assert request.headers["authorization"] == "Bearer inferenceapis-test-key" + assert body["model"] == INFERENCEAPIS_MODEL_ID + assert body["messages"] == [{"role": "user", "content": "Say hello"}] + assert body["max_tokens"] == 32 + assert response["content"][0]["text"] == "Hello from Inference APIs" + + +@pytest.mark.asyncio +async def test_inferenceapis_anthropic_messages_context_length_error_raises_context_window_exceeded(): + with respx.mock() as upstream: + upstream.post(f"{INFERENCEAPIS_API_BASE}/messages").respond( + 400, + json={ + "type": "error", + "error": { + "type": "invalid_request_error", + "message": "Input too long: 150055 input tokens, limit is 131072 for this model", + "code": "upstream_rejected", + }, + }, + ) + with pytest.raises(litellm.ContextWindowExceededError): + await litellm.anthropic.messages.acreate( + model=INFERENCEAPIS_MODEL, + messages=[{"role": "user", "content": "Say hello"}], + max_tokens=32, + api_key="inferenceapis-test-key", + client=AsyncHTTPHandler(transport=httpx.AsyncHTTPTransport()), + ) + + +def test_inferenceapis_chat_completion_cost_bills_cached_tokens_at_cache_read_rate(): + row: Final = litellm.model_cost[INFERENCEAPIS_MODEL] + with respx.mock() as upstream: + upstream.post(f"{INFERENCEAPIS_API_BASE}/chat/completions").respond( + 200, + json=_chat_completion_body( + { + "prompt_tokens": 1000, + "completion_tokens": 20, + "total_tokens": 1020, + "prompt_tokens_details": {"cached_tokens": 800}, + } + ), + ) + response: Final = litellm.completion( + model=INFERENCEAPIS_MODEL, + messages=[{"role": "user", "content": "Say hello"}], + api_key="inferenceapis-test-key", + ) + + assert row["cache_read_input_token_cost"] < row["input_cost_per_token"] + assert litellm.completion_cost(completion_response=response, model=INFERENCEAPIS_MODEL) == pytest.approx( + 200 * row["input_cost_per_token"] + 800 * row["cache_read_input_token_cost"] + 20 * row["output_cost_per_token"] + )